{
 "cells": [
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "%env LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n",
    "%env LLM_API_KEY=sk-替换为自己的Qwen API_KEY"
   ],
   "id": "74cf7942cfa0f0b8"
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "af375836-b870-458b-87d1-4e00565977eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T08:03:45.182629Z",
     "iopub.status.busy": "2024-12-04T08:03:45.182215Z",
     "iopub.status.idle": "2024-12-04T08:03:45.194354Z",
     "shell.execute_reply": "2024-12-04T08:03:45.193796Z",
     "shell.execute_reply.started": "2024-12-04T08:03:45.182608Z"
    },
    "papermill": {
     "duration": 0.115454,
     "end_time": "2024-11-23T14:29:00.919641",
     "exception": false,
     "start_time": "2024-11-23T14:29:00.804187",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%capture --no-stderr\n",
    "!pip install -U langchain langchain_community langchain_openai pypdf sentence_transformers chromadb shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1e2c72b8-ee12-4130-af88-699998aa230c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T08:03:45.195077Z",
     "iopub.status.busy": "2024-12-04T08:03:45.194900Z",
     "iopub.status.idle": "2024-12-04T08:03:45.401324Z",
     "shell.execute_reply": "2024-12-04T08:03:45.400807Z",
     "shell.execute_reply.started": "2024-12-04T08:03:45.195061Z"
    },
    "papermill": {
     "duration": 0.319981,
     "end_time": "2024-11-23T14:29:01.380771",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.060790",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "841d2b02-ad06-40d2-b11f-c7adccec6ca2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T08:03:45.402438Z",
     "iopub.status.busy": "2024-12-04T08:03:45.402258Z",
     "iopub.status.idle": "2024-12-04T08:03:45.405258Z",
     "shell.execute_reply": "2024-12-04T08:03:45.404837Z",
     "shell.execute_reply.started": "2024-12-04T08:03:45.402424Z"
    },
    "papermill": {
     "duration": 0.121409,
     "end_time": "2024-11-23T14:29:01.638126",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.516717",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "expr_version = 'split_01_1_markdown_text_split'\n",
    "\n",
    "preprocess_output_dir = os.path.join(os.path.pardir, 'outputs', 'v1_20240713')\n",
    "expr_dir = os.path.join(os.path.pardir, 'experiments', expr_version)\n",
    "\n",
    "os.makedirs(expr_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf7e81e3-4c82-4842-aef5-7592caaf1d39",
   "metadata": {
    "papermill": {
     "duration": 0.100379,
     "end_time": "2024-11-23T14:29:01.862379",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.762000",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 读取文档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e6920e29-bc7d-4635-be06-d151eaf0e100",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T08:03:45.405766Z",
     "iopub.status.busy": "2024-12-04T08:03:45.405646Z",
     "iopub.status.idle": "2024-12-04T08:03:45.927483Z",
     "shell.execute_reply": "2024-12-04T08:03:45.926994Z",
     "shell.execute_reply.started": "2024-12-04T08:03:45.405754Z"
    },
    "papermill": {
     "duration": 2.012298,
     "end_time": "2024-11-23T14:29:03.974974",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.962676",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "markdown_document = open(os.path.join(os.path.pardir, 'outputs', 'MinerU_parsed_20241204', '2024全球经济金融展望报告.md')).read()\n",
    "qa_df = pd.read_excel(os.path.join(preprocess_output_dir, 'question_answer.xlsx'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "841ec659-4ad7-4e1f-b1ea-3477bf97fde3",
   "metadata": {
    "papermill": {
     "duration": 0.100297,
     "end_time": "2024-11-23T14:29:04.219302",
     "exception": false,
     "start_time": "2024-11-23T14:29:04.119005",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 文档切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "74fe856a-7c19-4c3c-bb30-7abfa6298f74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T08:03:45.928055Z",
     "iopub.status.busy": "2024-12-04T08:03:45.927884Z",
     "iopub.status.idle": "2024-12-04T08:03:46.085142Z",
     "shell.execute_reply": "2024-12-04T08:03:46.084662Z",
     "shell.execute_reply.started": "2024-12-04T08:03:45.928042Z"
    },
    "papermill": {
     "duration": 0.109229,
     "end_time": "2024-11-23T14:29:04.429069",
     "exception": false,
     "start_time": "2024-11-23T14:29:04.319840",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from langchain.text_splitter import MarkdownTextSplitter\n",
    "from langchain.schema import Document\n",
    "\n",
    "def split_docs(markdown_document, chunk_size=500, chunk_overlap=50):\n",
    "    splitter = MarkdownTextSplitter(\n",
    "        chunk_size=chunk_size,\n",
    "        chunk_overlap=chunk_overlap\n",
    "    )\n",
    "    splitted_texts = splitter.split_text(markdown_document)\n",
    "    return [Document(page_content=text) for text in splitted_texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "47636d87-dfcc-4bc5-91d2-45c26a4e606d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T08:03:47.205714Z",
     "iopub.status.busy": "2024-12-04T08:03:47.204936Z",
     "iopub.status.idle": "2024-12-04T08:03:47.211754Z",
     "shell.execute_reply": "2024-12-04T08:03:47.211413Z",
     "shell.execute_reply.started": "2024-12-04T08:03:47.205642Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "92\n"
     ]
    }
   ],
   "source": [
    "splitted_docs = split_docs(markdown_document)\n",
    "print(len(splitted_docs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "543f6f4e-28c1-4238-ae99-9abab95c2318",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:12:08.535706Z",
     "iopub.status.busy": "2024-12-04T04:12:08.535585Z",
     "iopub.status.idle": "2024-12-04T04:12:08.551262Z",
     "shell.execute_reply": "2024-12-04T04:12:08.550818Z",
     "shell.execute_reply.started": "2024-12-04T04:12:08.535694Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(metadata={}, page_content='# 全球经济金融展望报告  \\n\\n2024年年报（总第57期）  \\n\\n报告日期：2023年12月12日  \\n\\n# 要点  \\n\\n●2023 年全球经济增长动力持续回落，各国复苏分化，发达经济体增速明显放缓，新兴经济体整体表现稳定。全球贸易增长乏力，各国生产景气度逐渐回落，内需对经济的拉动作用减弱。欧美央行货币政策紧缩态势放缓，美元指数高位震荡后走弱，全球股市表现总体好于预期，但区域分化明显。高利率环境抑制债券融资需求，债券违约风险持续上升。  \\n\\n$\\\\bullet$ 展望2024 年，预计全球经济复苏将依旧疲软，主要经济体增长态势和货币政策走势将进一步分化。欧美央行大概率结束本轮紧缩货币周期，美元指数将逐步走弱，流向新兴经济体的跨境资本将增加。国际原油市场短缺格局或延续，新能源发展成为重点。  \\n\\n$\\\\bullet$ 海湾六国经济发展与投资前景、高利率和高债务对美国房地产市场脆弱性的影响等热点问题值得关注。'),\n",
       " Document(metadata={}, page_content='![](images/c7e6ce1606712e84e07a05bcf6016906efa3fc778e40fcd0e91ac4fcb5503b79.jpg)  \\n主要经济体GDP 增速变化趋势（%）  \\n资料来源：IMF，中国银行研究院  \\n\\n# 中国银行研究院全球经济金融研究课题组  \\n\\n![](images/a5d0eb181c75231451c8f890ec50fe5822e2306a9beb543ca35a04880abbf639.jpg)  \\n\\n联系人：王有鑫  \\n电话：010-66594127  \\n邮件： wangyouxin_hq@bank-of-china.com  \\n\\n# 全球经济复苏疲软，货币政策取向分化  \\n\\n——中国银行全球经济金融展望报告（2024年）'),\n",
       " Document(metadata={}, page_content='2023年，全球经济增长动力持续回落。分区域看，各国复苏存在较大差异，发达经济体增速明显放缓，新兴经济体增速与2022年大致持平。生产端，全球供应链持续恢复，但生产景气度逐渐回落。需求端，内需对经济的拉动作用逐渐减弱，各国国内投资和跨境投资均持续承压；全球货物贸易量指数和价格指数下行，主要经济体出口贸易同比增速下降。欧美央行货币政策延续收紧态势，但步伐整体放缓；金融体系短期资金运行发生结构性变化，“去存款化”特征突出。美元指数高位震荡后走弱，全球股市表现总体好于预期，但区域分化显著。高利率环境抑制债券融资需求，债券违约风险持续上升，美国政府债务可持续性问题引发市场关注。展望2024年，预计全球经济复苏将依旧疲软，主要经济体增长态势和货币政策将进一步分化。欧美央行大概率结束本轮加息周期，日本央行可能退出负利率政策，跨境资本回流美国趋势将放缓，流向新兴经济体的资金将增加。美元指数将逐步走弱，新兴经济体货币汇率有望回升。国际原油市场短缺格局或延续，新能源发展成为重点。本期报告分别对海湾六国经济发展与投资前景、高利率和高债务对美国房地产市场脆弱性的影响两个专题展开分析。')]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splitted_docs[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fb9cf39-1221-4b46-ab92-b300dc261c8e",
   "metadata": {},
   "source": [
    "检查一下切分后的块长度分布"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c76b31aa-28af-430b-a62c-8879905176b7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:12:08.551916Z",
     "iopub.status.busy": "2024-12-04T04:12:08.551767Z",
     "iopub.status.idle": "2024-12-04T04:12:08.559387Z",
     "shell.execute_reply": "2024-12-04T04:12:08.558938Z",
     "shell.execute_reply.started": "2024-12-04T04:12:08.551904Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     92.000000\n",
       "mean     376.663043\n",
       "std      116.359171\n",
       "min       43.000000\n",
       "25%      328.750000\n",
       "50%      408.000000\n",
       "75%      466.250000\n",
       "max      499.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series([len(d.page_content) for d in splitted_docs]).describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "220dbc3a-fceb-4e49-a3f1-01e16660b2a6",
   "metadata": {
    "papermill": {
     "duration": 0.100209,
     "end_time": "2024-11-23T14:29:05.255871",
     "exception": false,
     "start_time": "2024-11-23T14:29:05.155662",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8598a11c-25d8-4af1-a98b-06a8c394e261",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:12:08.559931Z",
     "iopub.status.busy": "2024-12-04T04:12:08.559813Z",
     "iopub.status.idle": "2024-12-04T04:12:09.393241Z",
     "shell.execute_reply": "2024-12-04T04:12:09.392740Z",
     "shell.execute_reply.started": "2024-12-04T04:12:08.559919Z"
    },
    "papermill": {
     "duration": 0.989203,
     "end_time": "2024-11-23T14:29:06.345534",
     "exception": false,
     "start_time": "2024-11-23T14:29:05.356331",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device: cuda\n"
     ]
    }
   ],
   "source": [
    "from langchain.embeddings import HuggingFaceBgeEmbeddings\n",
    "from langchain_community.vectorstores import Chroma\n",
    "import torch\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "# device = 'cpu'\n",
    "print(f'device: {device}')\n",
    "\n",
    "def get_embeddings(model_path):\n",
    "    embeddings = HuggingFaceBgeEmbeddings(\n",
    "        model_name=model_path,\n",
    "        model_kwargs={'device': device},\n",
    "        encode_kwargs={'normalize_embeddings': True},\n",
    "        # show_progress=True\n",
    "        query_instruction='为这个句子生成表示以用于检索相关文章：'\n",
    "    )\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f6f46c73-7369-448f-a89a-ed3d817cad47",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:12:09.393971Z",
     "iopub.status.busy": "2024-12-04T04:12:09.393752Z",
     "iopub.status.idle": "2024-12-04T04:12:29.700659Z",
     "shell.execute_reply": "2024-12-04T04:12:29.698209Z",
     "shell.execute_reply.started": "2024-12-04T04:12:09.393957Z"
    },
    "papermill": {
     "duration": 83.983138,
     "end_time": "2024-11-23T14:35:06.117207",
     "exception": false,
     "start_time": "2024-11-23T14:33:42.134069",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import shutil\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "from langchain_community.vectorstores import Chroma\n",
    "\n",
    "model_path = 'BAAI/bge-large-zh-v1.5'\n",
    "embeddings = get_embeddings(model_path)\n",
    "\n",
    "persist_directory = os.path.join(expr_dir, 'chroma', 'bge')\n",
    "shutil.rmtree(persist_directory, ignore_errors=True)\n",
    "vector_db = Chroma.from_documents(\n",
    "    splitted_docs,\n",
    "    embedding=embeddings,\n",
    "    persist_directory=persist_directory\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7925564a-7d30-4914-baaf-4a00abb7686d",
   "metadata": {
    "papermill": {
     "duration": 0.109216,
     "end_time": "2024-11-23T14:35:26.464009",
     "exception": false,
     "start_time": "2024-11-23T14:35:26.354793",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 生成答案"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "27132c3b-0051-4df6-bf57-fd804acb8d17",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:12:29.704496Z",
     "iopub.status.busy": "2024-12-04T04:12:29.703387Z",
     "iopub.status.idle": "2024-12-04T04:12:29.808383Z",
     "shell.execute_reply": "2024-12-04T04:12:29.807932Z",
     "shell.execute_reply.started": "2024-12-04T04:12:29.704423Z"
    },
    "papermill": {
     "duration": 0.199165,
     "end_time": "2024-11-23T14:35:27.323500",
     "exception": false,
     "start_time": "2024-11-23T14:35:27.124335",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_511720/432419542.py:3: LangChainDeprecationWarning: The class `Ollama` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the :class:`~langchain-ollama package and should be used instead. To use it run `pip install -U :class:`~langchain-ollama` and import as `from :class:`~langchain_ollama import OllamaLLM``.\n",
      "  ollama_llm = Ollama(\n"
     ]
    }
   ],
   "source": [
    "from langchain.llms import Ollama\n",
    "\n",
    "ollama_llm = Ollama(\n",
    "    model='qwen2:7b-instruct-32k',\n",
    "    # model='qwen2:7b-instruct',\n",
    "    base_url='http://localhost:11434',\n",
    "    top_k=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "50404beb-3be0-4aaa-b124-8c7a52b84531",
   "metadata": {
    "editable": true,
    "execution": {
     "iopub.execute_input": "2024-12-04T04:12:29.809037Z",
     "iopub.status.busy": "2024-12-04T04:12:29.808907Z",
     "iopub.status.idle": "2024-12-04T04:12:29.812673Z",
     "shell.execute_reply": "2024-12-04T04:12:29.812295Z",
     "shell.execute_reply.started": "2024-12-04T04:12:29.809025Z"
    },
    "papermill": {
     "duration": 0.159318,
     "end_time": "2024-11-23T14:35:26.768506",
     "exception": false,
     "start_time": "2024-11-23T14:35:26.609188",
     "status": "completed"
    },
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def rag(llm, query, n_chunks=4):\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个金融分析师，擅长根据所获取的信息片段，对问题进行分析和推理。\n",
    "你的任务是根据所获取的信息片段（<<<<context>>><<<</context>>>之间的内容）回答问题。\n",
    "回答保持简洁，不必重复问题，不要添加描述性解释和与答案无关的任何内容。\n",
    "已知信息：\n",
    "<<<<context>>>\n",
    "{{knowledge}}\n",
    "<<<</context>>>\n",
    "\n",
    "问题：{{query}}\n",
    "请回答：\n",
    "\"\"\".strip()\n",
    "    chunks = vector_db.similarity_search(query, k=n_chunks)\n",
    "    prompt = prompt_tmpl.replace('{{knowledge}}', '\\n\\n'.join([doc.page_content for doc in chunks])).replace('{{query}}', query)\n",
    "    retry_count = 3\n",
    "\n",
    "    resp = ''\n",
    "    while retry_count > 0:\n",
    "        try:\n",
    "            resp = llm.invoke(prompt)\n",
    "            break\n",
    "        except Exception as e:\n",
    "            retry_count -= 1\n",
    "            sleeping_seconds = 2 ** (4 - retry_count)\n",
    "            print(f\"query={query}, error={e}, sleeping={sleeping_seconds}, remaining retry count={retry_count}\")\n",
    "            \n",
    "            time.sleep(sleeping_seconds)\n",
    "    \n",
    "    return resp, chunks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95e5a804-2dc6-411c-ba71-6ccf765b2b73",
   "metadata": {
    "papermill": {
     "duration": 0.135973,
     "end_time": "2024-11-23T14:35:27.001401",
     "exception": false,
     "start_time": "2024-11-23T14:35:26.865428",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "166392d8-f801-4372-b8ad-3e79aef0b350",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:12:29.813207Z",
     "iopub.status.busy": "2024-12-04T04:12:29.813082Z",
     "iopub.status.idle": "2024-12-04T04:12:29.822272Z",
     "shell.execute_reply": "2024-12-04T04:12:29.821747Z",
     "shell.execute_reply.started": "2024-12-04T04:12:29.813195Z"
    },
    "papermill": {
     "duration": 0.141864,
     "end_time": "2024-11-23T14:35:27.564409",
     "exception": false,
     "start_time": "2024-11-23T14:35:27.422545",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "prediction_df = qa_df[qa_df['dataset'] == 'test'][['uuid', 'question', 'qa_type', 'answer']].rename(columns={'answer': 'ref_answer'})\n",
    "\n",
    "def predict(llm, prediction_df, n_chunks):\n",
    "    prediction_df = prediction_df.copy()\n",
    "    answer_dict = {}\n",
    "\n",
    "    for idx, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        uuid = row['uuid']\n",
    "        question = row['question']\n",
    "        answer, chunks = rag(llm, question, n_chunks=n_chunks)\n",
    "        assert len(chunks) <= n_chunks\n",
    "        answer_dict[question] = {\n",
    "            'uuid': uuid,\n",
    "            'ref_answer': row['ref_answer'],\n",
    "            'gen_answer': answer,\n",
    "            'chunks': chunks\n",
    "        }\n",
    "\n",
    "    prediction_df.loc[:, 'gen_answer'] = prediction_df['question'].apply(lambda q: answer_dict[q]['gen_answer'])\n",
    "    prediction_df.loc[:, 'chunks'] = prediction_df['question'].apply(lambda q: answer_dict[q]['chunks'])\n",
    "\n",
    "    return prediction_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ca46d5f1-e698-457d-abb6-92d83cd59c66",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:12:29.822904Z",
     "iopub.status.busy": "2024-12-04T04:12:29.822746Z",
     "iopub.status.idle": "2024-12-04T04:16:26.790820Z",
     "shell.execute_reply": "2024-12-04T04:16:26.790394Z",
     "shell.execute_reply.started": "2024-12-04T04:12:29.822892Z"
    },
    "papermill": {
     "duration": 514.92352,
     "end_time": "2024-11-23T14:44:02.805529",
     "exception": false,
     "start_time": "2024-11-23T14:35:27.882009",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa072dae17674d3bbdcf805435441dd8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_df = predict(ollama_llm, prediction_df, n_chunks=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f7026bac-9927-4a33-85c0-bc1b35f3a603",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:16:26.791475Z",
     "iopub.status.busy": "2024-12-04T04:16:26.791342Z",
     "iopub.status.idle": "2024-12-04T04:16:26.982855Z",
     "shell.execute_reply": "2024-12-04T04:16:26.980421Z",
     "shell.execute_reply.started": "2024-12-04T04:16:26.791462Z"
    }
   },
   "outputs": [],
   "source": [
    "save_path = os.path.join(expr_dir, 'predictions.xlsx')\n",
    "pred_df.to_excel(save_path, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d79e974-089f-4c08-ba5e-804f6542e06a",
   "metadata": {
    "papermill": {
     "duration": 0.14423,
     "end_time": "2024-11-23T14:44:03.513124",
     "exception": false,
     "start_time": "2024-11-23T14:44:03.368894",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "217568fe-c0e4-49eb-9a7c-9fdfbc033d8a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:16:26.986337Z",
     "iopub.status.busy": "2024-12-04T04:16:26.985599Z",
     "iopub.status.idle": "2024-12-04T04:16:27.241760Z",
     "shell.execute_reply": "2024-12-04T04:16:27.241247Z",
     "shell.execute_reply.started": "2024-12-04T04:16:26.986268Z"
    },
    "papermill": {
     "duration": 0.369729,
     "end_time": "2024-11-23T14:44:04.017198",
     "exception": false,
     "start_time": "2024-11-23T14:44:03.647469",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "import time\n",
    "\n",
    "judge_llm = ChatOpenAI(\n",
    "    api_key=os.environ['LLM_API_KEY'],\n",
    "    base_url=os.environ['LLM_BASE_URL'],\n",
    "    model_name='qwen2-72b-instruct',\n",
    "    temperature=0\n",
    ")\n",
    "\n",
    "def evaluate(prediction_df):\n",
    "    \"\"\"\n",
    "    对预测结果进行打分\n",
    "    :param prediction_df: 预测结果，需要包含问题，参考答案，生成的答案，列名分别为question, ref_answer, gen_answer\n",
    "    :return 打分模型原始返回结果\n",
    "    \"\"\"\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个经济学博士，现在我有一系列问题，有一个助手已经对这些问题进行了回答，你需要参照参考答案，评价这个助手的回答是否正确，仅回复“是”或“否”即可，不要带其他描述性内容或无关信息。\n",
    "问题：\n",
    "<question>\n",
    "{{question}}\n",
    "</question>\n",
    "\n",
    "参考答案：\n",
    "<ref_answer>\n",
    "{{ref_answer}}\n",
    "</ref_answer>\n",
    "\n",
    "助手回答：\n",
    "<gen_answer>\n",
    "{{gen_answer}}\n",
    "</gen_answer>\n",
    "请评价：\n",
    "    \"\"\"\n",
    "    results = []\n",
    "\n",
    "    for _, row in prediction_df.iterrows():\n",
    "        question = row['question']\n",
    "        ref_answer = row['ref_answer']\n",
    "        gen_answer = row['gen_answer']\n",
    "\n",
    "        prompt = prompt_tmpl.replace('{{question}}', question).replace('{{ref_answer}}', str(ref_answer)).replace('{{gen_answer}}', gen_answer).strip()\n",
    "        \n",
    "        retry_count = 3\n",
    "        result = ''\n",
    "        \n",
    "        while retry_count > 0:\n",
    "            try:\n",
    "                result = judge_llm.invoke(prompt).content\n",
    "                break\n",
    "            except Exception as e:\n",
    "                retry_count -= 1\n",
    "                sleeping_seconds = 2 ** (4 - retry_count)\n",
    "                print(f\"query={query}, error={e}, sleeping={sleeping_seconds}, remaining retry count={retry_count}\")\n",
    "                \n",
    "                time.sleep(sleeping_seconds)\n",
    "        \n",
    "        results.append(result)\n",
    "\n",
    "        time.sleep(1)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "71db81af-b8f9-47ba-958b-761896516605",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:16:27.242295Z",
     "iopub.status.busy": "2024-12-04T04:16:27.242168Z",
     "iopub.status.idle": "2024-12-04T04:18:59.831171Z",
     "shell.execute_reply": "2024-12-04T04:18:59.828722Z",
     "shell.execute_reply.started": "2024-12-04T04:16:27.242283Z"
    },
    "papermill": {
     "duration": 150.566109,
     "end_time": "2024-11-23T14:46:34.714324",
     "exception": false,
     "start_time": "2024-11-23T14:44:04.148215",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pred_df['raw_score'] = evaluate(pred_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7da1b98e-99aa-4e11-9297-91eac1c62493",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:18:59.834757Z",
     "iopub.status.busy": "2024-12-04T04:18:59.834005Z",
     "iopub.status.idle": "2024-12-04T04:18:59.840597Z",
     "shell.execute_reply": "2024-12-04T04:18:59.840196Z",
     "shell.execute_reply.started": "2024-12-04T04:18:59.834688Z"
    },
    "papermill": {
     "duration": 0.138037,
     "end_time": "2024-11-23T14:46:35.040595",
     "exception": false,
     "start_time": "2024-11-23T14:46:34.902558",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['是', '否'], dtype=object)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df['raw_score'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2c99c078-d294-40b8-b57b-31cfd7349c3e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:18:59.841260Z",
     "iopub.status.busy": "2024-12-04T04:18:59.841040Z",
     "iopub.status.idle": "2024-12-04T04:18:59.854108Z",
     "shell.execute_reply": "2024-12-04T04:18:59.853555Z",
     "shell.execute_reply.started": "2024-12-04T04:18:59.841247Z"
    },
    "papermill": {
     "duration": 0.107466,
     "end_time": "2024-11-23T14:46:35.243603",
     "exception": false,
     "start_time": "2024-11-23T14:46:35.136137",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pred_df['score'] = (pred_df['raw_score'] == '是').astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "423897f2-786e-415b-a613-55a4359faf76",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:18:59.854853Z",
     "iopub.status.busy": "2024-12-04T04:18:59.854694Z",
     "iopub.status.idle": "2024-12-04T04:18:59.860410Z",
     "shell.execute_reply": "2024-12-04T04:18:59.859960Z",
     "shell.execute_reply.started": "2024-12-04T04:18:59.854838Z"
    },
    "papermill": {
     "duration": 0.094328,
     "end_time": "2024-11-23T14:46:35.431162",
     "exception": false,
     "start_time": "2024-11-23T14:46:35.336834",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.72"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df['score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "79325429-9cf1-4e2c-95ac-cb0c1a3b6156",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:18:59.862905Z",
     "iopub.status.busy": "2024-12-04T04:18:59.862718Z",
     "iopub.status.idle": "2024-12-04T04:19:00.045792Z",
     "shell.execute_reply": "2024-12-04T04:19:00.043351Z",
     "shell.execute_reply.started": "2024-12-04T04:18:59.862886Z"
    },
    "papermill": {
     "duration": 0.289336,
     "end_time": "2024-11-23T14:46:35.804651",
     "exception": false,
     "start_time": "2024-11-23T14:46:35.515315",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "save_path = os.path.join(expr_dir, 'eval_df.xlsx')\n",
    "pred_df.to_excel(save_path, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81702255-cdac-4b60-9629-1fccdb3f575b",
   "metadata": {},
   "source": [
    "# 结果分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "88fc7227-9c21-48da-b179-5070406eb113",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:19:00.048569Z",
     "iopub.status.busy": "2024-12-04T04:19:00.047847Z",
     "iopub.status.idle": "2024-12-04T04:19:00.060784Z",
     "shell.execute_reply": "2024-12-04T04:19:00.060429Z",
     "shell.execute_reply.started": "2024-12-04T04:19:00.048500Z"
    },
    "papermill": {
     "duration": 0.088622,
     "end_time": "2024-11-23T14:46:36.016801",
     "exception": false,
     "start_time": "2024-11-23T14:46:35.928179",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pred_df['avg_chunk_len'] = pred_df['chunks'].apply(lambda chunks: sum([len(d.page_content) for d in chunks]) / len(chunks))\n",
    "pred_df['max_chunk_len'] = pred_df['chunks'].apply(lambda chunks: max([len(d.page_content) for d in chunks]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fb1a3e72-cb28-419a-8f6c-a0ef5d34c67a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:19:00.061361Z",
     "iopub.status.busy": "2024-12-04T04:19:00.061228Z",
     "iopub.status.idle": "2024-12-04T04:19:00.505682Z",
     "shell.execute_reply": "2024-12-04T04:19:00.505194Z",
     "shell.execute_reply.started": "2024-12-04T04:19:00.061348Z"
    }
   },
   "outputs": [],
   "source": [
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee703234-958f-4a37-a830-cf24761a70c2",
   "metadata": {},
   "source": [
    "从平均上下文长度来看，回答错误的问题对应的上下文，方差更大"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ff2413c8-f7a7-4e70-9178-c42ff2427426",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:19:00.506375Z",
     "iopub.status.busy": "2024-12-04T04:19:00.506113Z",
     "iopub.status.idle": "2024-12-04T04:19:00.594828Z",
     "shell.execute_reply": "2024-12-04T04:19:00.594406Z",
     "shell.execute_reply.started": "2024-12-04T04:19:00.506361Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='score', ylabel='avg_chunk_len'>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAta0lEQVR4nO3df1xUdaL/8ffwawaQwYBgNPDHapshmI/UjOp6Sw2WqIys7Rum1HbryqKZtF7jLluulXi9m2mbldu21WOT2NvevO1aPJRIcQ0sr0WalptuBYYDriWj1IDAfP8w59Fctdhh4AyH1/PxmMeD+Zwzc94nwnk/zufMORaPx+MRAACASYUYHQAAAKA3UXYAAICpUXYAAICpUXYAAICpUXYAAICpUXYAAICpUXYAAICphRkdIBh0dXWpsbFRMTExslgsRscBAADd4PF4dOzYMQ0dOlQhIWc/fkPZkdTY2KiUlBSjYwAAAD80NDQoOTn5rMspO5JiYmIknfyPZbfbDU4DAAC6w+VyKSUlxfs5fjaUHck7dWW32yk7AAD0M993CgonKAMAAFOj7AAAAFOj7AAAAFOj7AAAAFOj7AAAAFOj7AAAAFOj7AAAAFOj7AAAAFOj7AAAAFOj7AAAAFOj7AAAAFOj7AAAAFPjRqAAMMB5PB653W6jY/SYx+NRW1ubJMlqtX7vzSGDnc1m6/f7ECwoOwAwwLndbmVnZxsdA/9HRUWFIiMjjY5hCkxjAQAAU+PIDgAMcDabTRUVFUbH6DG3263c3FxJ0vr162Wz2QxO1DP9PX8woewAwABnsVhMN11is9lMt0/wH9NYAADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1MKMDoD+wePxyO12Gx2jxzwej9ra2iRJVqtVFovF4EQ9Y7PZ+v0+AEBvo+ygW9xut7Kzs42Ogf+joqJCkZGRRscAgKDGNBYAADA1juygW2w2myoqKoyO0WNut1u5ubmSpPXr18tmsxmcqGf6e34A6AuUHXSLxWIx3XSJzWYz3T4BAE7HNBYAADA1yg4AADA1yg4AADA1yg4AADA1yg4AADA1yg4AADA1yg4AADA1yg4AADA1yg4AADC1oCk7y5cvl8Vi0b333usdu/LKK2WxWHwec+fO9XldfX29cnJyFBUVpcTERC1atEgdHR19nB4AAASroLhdxI4dO7R27VqNGzfutGV33XWXli5d6n0eFRXl/bmzs1M5OTlyOByqqanRoUOHNGfOHIWHh2vZsmV9kh0AAAQ3w4/sHD9+XLNmzdIzzzyjc84557TlUVFRcjgc3ofdbvcu27Rpk/bu3asXX3xR48ePV3Z2th566CGtWbNG7e3tZ91mW1ubXC6XzwMAAJiT4WWnsLBQOTk5mj59+hmXr1u3TgkJCUpLS1NxcbG++uor77La2lqlp6crKSnJO5aVlSWXy6U9e/acdZulpaWKjY31PlJSUgK3QwAAIKgYOo1VXl6ud999Vzt27Djj8ry8PA0fPlxDhw7Vrl27tHjxYu3bt0+vvPKKJMnpdPoUHUne506n86zbLS4uVlFRkfe5y+Wi8AAAYFKGlZ2GhgYtWLBAlZWVstlsZ1zn7rvv9v6cnp6uIUOGaNq0aTpw4IBGjRrl97atVqusVqvfrwcAAP2HYdNYO3fuVHNzsy6++GKFhYUpLCxM1dXVevzxxxUWFqbOzs7TXjN58mRJ0v79+yVJDodDTU1NPuuceu5wOHp5DwAAQH9gWNmZNm2adu/erbq6Ou9j4sSJmjVrlurq6hQaGnraa+rq6iRJQ4YMkSRlZGRo9+7dam5u9q5TWVkpu92u1NTUPtkPAAAQ3AybxoqJiVFaWprPWHR0tOLj45WWlqYDBw6orKxM11xzjeLj47Vr1y4tXLhQU6ZM8X5FPTMzU6mpqZo9e7ZWrFghp9OpkpISFRYWMk0FAAAkBcl1ds4kIiJCb7zxhlatWqXW1lalpKRo5syZKikp8a4TGhqqDRs2qKCgQBkZGYqOjlZ+fr7PdXkAAMDAFlRlZ8uWLd6fU1JSVF1d/b2vGT58uF5//fVeTAUAAPozw6+zAwAA0JsoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNTCjA5gdh6PR2632+gY+Ma3fxf8XoKHzWaTxWIxOgYAk6Ls9DK3263s7GyjY+AMcnNzjY6Ab1RUVCgyMtLoGABMimksAABgahzZ6UPHx98qTwj/yQ3l8UhdHSd/DgmTmDoxjKWrQ4PqXjI6BoABgE/ePuQJCZNCw42OAUUYHQCSPEYHADBgMI0FAABMjbIDAABMjbIDAABMjbIDAABMjbIDAABMjbIDAABMjbIDAABMjbIDAABMjbIDAABMjbIDAABMjbIDAABMjbIDAABMjRuBAoCfPB6P3G630THwjW//Lvi9BA+bzSaLxWJoBsoOAPjJ7XYrOzvb6Bg4g9zcXKMj4BsVFRWKjIw0NAPTWAAAwNQ4sgMAAfDEFV/IGuoxOsaA5vFI7V0nf44IkQyeORnQ2jotmrctzugYXpQdAAgAa6hH1lCjU8BmdAB8I7iKP9NYAADA1Cg7AADA1Cg7AADA1IKm7CxfvlwWi0X33nuvd8ztdquwsFDx8fEaNGiQZs6cqaamJp/X1dfXKycnR1FRUUpMTNSiRYvU0dHRx+kBAECwCoqys2PHDq1du1bjxo3zGV+4cKH+/Oc/6+WXX1Z1dbUaGxt14403epd3dnYqJydH7e3tqqmp0QsvvKDnn39eDzzwQF/vAgAACFKGl53jx49r1qxZeuaZZ3TOOed4x1taWvTss89q5cqVmjp1qiZMmKDnnntONTU12r59uyRp06ZN2rt3r1588UWNHz9e2dnZeuihh7RmzRq1t7cbtUsAACCIGF52CgsLlZOTo+nTp/uM79y5UydOnPAZHzNmjIYNG6ba2lpJUm1trdLT05WUlORdJysrSy6XS3v27DnrNtva2uRyuXweAADAnAy9zk55ebneffdd7dix47RlTqdTERERGjx4sM94UlKSnE6nd51vF51Ty08tO5vS0lL98pe/7GF6AADQHxh2ZKehoUELFizQunXrZLP17WWgiouL1dLS4n00NDT06fYBAEDfMazs7Ny5U83Nzbr44osVFhamsLAwVVdX6/HHH1dYWJiSkpLU3t6uo0eP+ryuqalJDodDkuRwOE77dtap56fWOROr1Sq73e7zAAAA5mRY2Zk2bZp2796turo672PixImaNWuW9+fw8HBVVVV5X7Nv3z7V19crIyNDkpSRkaHdu3erubnZu05lZaXsdrtSU1P7fJ8AAEDwMeycnZiYGKWlpfmMRUdHKz4+3jt+5513qqioSHFxcbLb7Zo/f74yMjJ06aWXSpIyMzOVmpqq2bNna8WKFXI6nSopKVFhYaGsVmuf7xMAAAg+QX0j0Mcee0whISGaOXOm2tralJWVpSeffNK7PDQ0VBs2bFBBQYEyMjIUHR2t/Px8LV261MDUAAAgmARV2dmyZYvPc5vNpjVr1mjNmjVnfc3w4cP1+uuv93IyAADQXxl+nR0AAIDeRNkBAACmRtkBAACmRtkBAACmRtkBAACmRtkBAACmRtkBAACmRtkBAACmFlQXFQSA/qqt0+gEQPAItr8Hyg4ABMC8bfFGRwBwFkxjAQAAU+PIDgAEwBNXHJE11OgUQHBo6wyuo52UHQAIAGuoKDtAkGIaCwAAmBpHdvpS5wmjEwDBg78HAH2EstOHYt4vNzoCAAADDtNYAADA1Diy04eOXfT/pNBwo2MAwaHzBEc7AfQJyk5fCg2n7AAA0MeYxgIAAKZG2QEAAKbm1zRWZ2ennn/+eVVVVam5uVldXV0+y998882AhAMAAOgpv8rOggUL9PzzzysnJ0dpaWmyWCyBzgUAABAQfpWd8vJy/dd//ZeuueaaQOcBAAAIKL/O2YmIiNDo0aMDnQUAACDg/Co79913n1avXi2PxxPoPAAAAAHl1zTWtm3btHnzZlVUVGjs2LEKD/e9dswrr7wSkHAAAAA95VfZGTx4sHJzcwOdBQAAIOD8KjvPPfdcoHMAAAD0Cr8vKtjR0aE33nhDa9eu1bFjxyRJjY2NOn78eMDCAQAA9JRfR3Y+++wz/ehHP1J9fb3a2tp09dVXKyYmRv/xH/+htrY2Pf3004HOCQAA4Be/juwsWLBAEydO1JdffqnIyEjveG5urqqqqgIWDgAAoKf8OrLzl7/8RTU1NYqIiPAZHzFihD7//POABAMAAAgEv47sdHV1qbOz87TxgwcPKiYmpsehAAAAAsWvspOZmalVq1Z5n1ssFh0/flwPPvggt5AAAABBxa9prEcffVRZWVlKTU2V2+1WXl6ePv74YyUkJOill14KdEYAAAC/+VV2kpOT9f7776u8vFy7du3S8ePHdeedd2rWrFk+JywDAAAYza+yI0lhYWG67bbbApkFAAAg4Lpddv70pz91+02vv/56v8IAAAAEWrfLzg033NCt9SwWyxm/qQUAAGCEbpedrq6u3swBAADQK/y+N1Z3pKenq6GhoTc3AQAA8J16tex8+umnOnHiRG9uAgAA4Dv1atkBAAAwmqFl56mnntK4ceNkt9tlt9uVkZGhiooK7/Irr7xSFovF5zF37lyf96ivr1dOTo6ioqKUmJioRYsWqaOjo693BQAABCm/r7MTCMnJyVq+fLnOP/98eTwevfDCC5oxY4bee+89jR07VpJ01113aenSpd7XREVFeX/u7OxUTk6OHA6HampqdOjQIc2ZM0fh4eFatmxZn+8PAAAIPoaWneuuu87n+SOPPKKnnnpK27dv95adqKgoORyOM75+06ZN2rt3r9544w0lJSVp/Pjxeuihh7R48WItWbLktLuyn9LW1qa2tjbvc5fLFaA9AgAAwSZoztnp7OxUeXm5WltblZGR4R1ft26dEhISlJaWpuLiYn311VfeZbW1tUpPT1dSUpJ3LCsrSy6XS3v27DnrtkpLSxUbG+t9pKSk9M5OAQAAw/lVdg4ePHjWZdu3b/f+vHbtWp8icia7d+/WoEGDZLVaNXfuXK1fv16pqamSpLy8PL344ovavHmziouL9fvf/97nFhVOp/O09z/13Ol0nnWbxcXFamlp8T74ejwAAObl1zRWZmamtm3bpri4OJ/xt956Szk5OTp69Kikk2Xl+1xwwQWqq6tTS0uL/vjHPyo/P1/V1dVKTU3V3Xff7V0vPT1dQ4YM0bRp03TgwAGNGjXKn+iSJKvVKqvV6vfrAQBA/+FX2bn00kuVmZmpzZs3KyYmRpK0detWXXfddVqyZMk/9F4REREaPXq0JGnChAnasWOHVq9erbVr15627uTJkyVJ+/fv16hRo+RwOPTOO+/4rNPU1CRJZz3PBwB6Q1unRZLH6BgDmscjtX9zsf+IEMliMTbPQHby7yF4+FV2fvvb3+qmm27Sddddp40bN6qmpkbXX3+9Hn74YS1YsKBHgbq6unxOHv62uro6SdKQIUMkSRkZGXrkkUfU3NysxMRESVJlZaXsdrt3KgwA+sK8bXHfvxIAQ/hVdkJCQlReXq6cnBxNnTpVu3btUmlpqebNm/cPvU9xcbGys7M1bNgwHTt2TGVlZdqyZYs2btyoAwcOqKysTNdcc43i4+O1a9cuLVy4UFOmTNG4ceMknZxOS01N1ezZs7VixQo5nU6VlJSosLCQaSoAACDpHyg7u3btOm1syZIluvXWW3XbbbdpypQp3nVOlZHv09zcrDlz5ujQoUOKjY3VuHHjtHHjRl199dVqaGjQG2+8oVWrVqm1tVUpKSmaOXOmSkpKvK8PDQ3Vhg0bVFBQoIyMDEVHRys/P9/nujwA0FtsNpvPhVBhLLfbrdzcXEnS+vXrZbPZDE4ESUHxe7B4PJ5uTTKHhITIYrHo26t/+/mpny0Wizo7O3snbS9xuVyKjY1VS0uL7HZ7QN/766+/VnZ2tiTp2MWzpdDwgL4/0G91nlDMu7+XJFVUVCgyMtLgQOjvvv3vLf9PDQzd/fzu9pGdTz75JCDBAAAA+lK3y87w4cN7MwcAAECv8Pt2ER9//LE2b96s5uZmdXV1+Sx74IEHehwMAAAgEPwqO88884wKCgqUkJAgh8Mhy7cuZmCxWCg7AAAgaPhVdh5++GE98sgjWrx4caDzAAAABJRf98b68ssvdfPNNwc6CwAAQMD5VXZuvvlmbdq0KdBZAAAAAs6vaazRo0frF7/4hbZv36709HSFh/teO+aee+4JSDgAAICe8qvs/OY3v9GgQYNUXV2t6upqn2UWi4WyAwAAgoZfZYcLDAIAgP7Cr3N2AAAA+gu/juz85Cc/+c7lv/vd7/wKAwAAEGh+lZ0vv/zS5/mJEyf0wQcf6OjRo5o6dWpAggEAAASCX2Vn/fr1p411dXWpoKBAo0aN6nEoAACAQAnYOTshISEqKirSY489Fqi3BAAA6LGAnqB84MABdXR0BPItAQAAesSvaayioiKf5x6PR4cOHdJrr72m/Pz8gAQDAAAIBL/KznvvvefzPCQkROeee64effTR7/2m1kBm6eqQx+gQA53HI3V9c/QxJEyyWIzNM4BZujgKDKBv+FV2Nm/eHOgcA8KgupeMjgAAwIDDRQUBAICp+XVkp6mpST/72c9UVVWl5uZmeTy+kzOdnZ0BCWcGNptNFRUVRsfAN9xut3JzcyWdvISCzWYzOBEk8XsA0Kv8Kju333676uvr9Ytf/EJDhgyRhfMezspisSgyMtLoGDgDm83G7wYABgC/ys62bdv0l7/8RePHjw9wHAAAgMDy65ydlJSU06auAAAAgpFfZWfVqlW6//779emnnwY4DgAAQGB1exrrnHPO8Tk3p7W1VaNGjVJUVJTCw8N91v3iiy8ClxAAAKAHul12Vq1a1YsxAAAAeke3yw63gQAAAP2RX+fsvP7669q4ceNp45s2beKaMgAAIKj4VXbuv//+M144sKurS/fff3+PQwEAAASKX2Xn448/Vmpq6mnjY8aM0f79+3scCgAAIFD8KjuxsbH629/+dtr4/v37FR0d3eNQAAAAgeJX2ZkxY4buvfdeHThwwDu2f/9+3Xfffbr++usDFg4AAKCn/Co7K1asUHR0tMaMGaORI0dq5MiRuvDCCxUfH69f/epXgc4IAADgN7/ujRUbG6uamhpVVlbq/fffV2RkpMaNG6cpU6YEOh8AAECP+FV2pJN3887MzFRmZuZZ10lPT9frr7+ulJQUfzcDAADQI35NY3XXp59+qhMnTvTmJgAAAL5Tr5YdAAAAo1F2AACAqVF2AACAqVF2AACAqVF2AACAqfVq2Vm7dq2SkpJ6cxMAAADfya/r7Dz++ONnHLdYLLLZbBo9erSmTJmivLy873yfp556Sk899ZQ+/fRTSdLYsWP1wAMPKDs7W5Lkdrt13333qby8XG1tbcrKytKTTz7pU6Dq6+tVUFCgzZs3a9CgQcrPz1dpaanCwvy+hBAAADARvxrBY489psOHD+urr77SOeecI0n68ssvFRUVpUGDBqm5uVk/+MEPtHnz5u+8oGBycrKWL1+u888/Xx6PRy+88IJmzJih9957T2PHjtXChQv12muv6eWXX1ZsbKzmzZunG2+8UW+99ZYkqbOzUzk5OXI4HKqpqdGhQ4c0Z84chYeHa9myZf7sGgAAMBm/prGWLVumSZMm6eOPP9aRI0d05MgR/fWvf9XkyZO1evVq1dfXy+FwaOHChd/5Ptddd52uueYanX/++frhD3+oRx55RIMGDdL27dvV0tKiZ599VitXrtTUqVM1YcIEPffcc6qpqdH27dslSZs2bdLevXv14osvavz48crOztZDDz2kNWvWqL293Z9dAwAAJuNX2SkpKdFjjz2mUaNGecdGjx6tX/3qVyouLlZycrJWrFjhPQLTHZ2dnSovL1dra6syMjK0c+dOnThxQtOnT/euM2bMGA0bNky1tbWSpNraWqWnp/tMa2VlZcnlcmnPnj1n3VZbW5tcLpfPAwAAmJNfZefQoUPq6Og4bbyjo0NOp1OSNHToUB07dux732v37t0aNGiQrFar5s6dq/Xr1ys1NVVOp1MREREaPHiwz/pJSUnebTidztNOgD71/NQ6Z1JaWqrY2Fjvg3t3AQBgXn6Vnauuukr/+q//qvfee8879t5776mgoEBTp06VdLLEjBw58nvf64ILLlBdXZ3efvttFRQUKD8/X3v37vUnVrcVFxerpaXF+2hoaOjV7QEAAOP4VXaeffZZxcXFacKECbJarbJarZo4caLi4uL07LPPSpIGDRqkRx999HvfKyIiQqNHj9aECRNUWlqqiy66SKtXr5bD4VB7e7uOHj3qs35TU5McDockyeFwqKmp6bTlp5adjdVqld1u93kAAABz8uvbWA6HQ5WVlfroo4/017/+VdLJIzQXXHCBd52rrrrKr0BdXV1qa2vThAkTFB4erqqqKs2cOVOStG/fPtXX1ysjI0OSlJGRoUceeUTNzc1KTEyUJFVWVsputys1NdWv7QMAAHPxq+xs27ZNV1xxhcaMGaMxY8b4vfHi4mJlZ2dr2LBhOnbsmMrKyrRlyxZt3LhRsbGxuvPOO1VUVKS4uDjZ7XbNnz9fGRkZuvTSSyVJmZmZSk1N1ezZs7VixQo5nU6VlJSosLBQVqvV71wAAMA8/Co7U6dO1Xnnnadbb71Vt912m99HUZqbmzVnzhwdOnRIsbGxGjdunDZu3Kirr75a0snr+YSEhGjmzJk+FxU8JTQ0VBs2bFBBQYEyMjIUHR2t/Px8LV261K88AADAfCwej8fzj77o73//u8rLy/XSSy+ptrZW48aN06xZs3TrrbcqOTm5N3L2KpfLpdjYWLW0tHD+jsl9/fXX3it0V1RUKDIy0uBEAAKFv++Bp7uf336doJyQkKB58+bprbfe0oEDB3TzzTfrhRde0IgRI7zfxgIAAAgGPb4R6MiRI3X//fdr+fLlSk9PV3V1dSByAQAABESPys5bb72ln/70pxoyZIjy8vKUlpam1157LVDZAAAAesyvE5SLi4tVXl6uzz//XJmZmVq9erVmzJihqKioQOcDAADoEb/KztatW7Vo0SL9+Mc/VkJCQqAzAQAABIxfZefUDT737t2r//3f/z3tDuPXX399z5MBAAAEgF9l55NPPlFubq527doli8WiU99et1gskk7ewRwAACAY+HWC8j333KMRI0aoublZUVFR2rNnj7Zu3aqJEydqy5YtAY4IAADgP7+O7NTW1urNN99UQkKCQkJCFBISoiuuuEKlpaW65557fO6GDgAAYCS/yk5nZ6diYmIknbzAYGNjoy644AINHz5c+/btC2hAAEDv8ng8crvdRsfosW/vgxn2x2azeU8PQc/4VXbS0tL0/vvva+TIkZo8ebJWrFihiIgI/eY3v9EPfvCDQGcEAPQit9vtvc2CWeTm5hodoce45UXg+FV2SkpK1NraKklaunSprr32Wv3TP/2T4uPj9Yc//CGgAQEAAHrCr7KTlZXl/Xn06NH66KOP9MUXX+icc87hkBsA9DM2m00VFRVGx+gxj8ejtrY2SZLVau33n0c2m83oCKbhV9k5k7i4uEC9FQCgD1ksFtNMl3Alf5xJj28ECgAAEMwoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNTCjA6A/sHj8cjtdhsdo8e+vQ9m2B+bzSaLxWJ0DAAIapQddIvb7VZ2drbRMQIqNzfX6Ag9VlFRocjISKNjAEBQYxoLAACYGkd20C02m00VFRVGx+gxj8ejtrY2SZLVau33U0A2m83oCAAQ9Cg76BaLxWKa6ZKoqCijIwAA+hDTWAAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQoOwAAwNQMLTulpaWaNGmSYmJilJiYqBtuuEH79u3zWefKK6+UxWLxecydO9dnnfr6euXk5CgqKkqJiYlatGiROjo6+nJXAABAkDL0OjvV1dUqLCzUpEmT1NHRoX//939XZmam9u7dq+joaO96d911l5YuXep9/u3rpHR2dionJ0cOh0M1NTU6dOiQ5syZo/DwcC1btqxP9wcAAAQfi8fj8Rgd4pTDhw8rMTFR1dXVmjJliqSTR3bGjx+vVatWnfE1FRUVuvbaa9XY2KikpCRJ0tNPP63Fixfr8OHDioiI+N7tulwuxcbGqqWlRXa7PWD7AwAAek93P7+D6pydlpYWSVJcXJzP+Lp165SQkKC0tDQVFxfrq6++8i6rra1Venq6t+hIUlZWllwul/bs2XPG7bS1tcnlcvk8AACAOQXN7SK6urp077336vLLL1daWpp3PC8vT8OHD9fQoUO1a9cuLV68WPv27dMrr7wiSXI6nT5FR5L3udPpPOO2SktL9ctf/rKX9gQAAASToCk7hYWF+uCDD7Rt2zaf8bvvvtv7c3p6uoYMGaJp06bpwIEDGjVqlF/bKi4uVlFRkfe5y+VSSkqKf8EBAEBQC4pprHnz5mnDhg3avHmzkpOTv3PdyZMnS5L2798vSXI4HGpqavJZ59Rzh8NxxvewWq2y2+0+DwAAYE6Glh2Px6N58+Zp/fr1evPNNzVy5MjvfU1dXZ0kaciQIZKkjIwM7d69W83Nzd51KisrZbfblZqa2iu5AQBA/2HoNFZhYaHKysr06quvKiYmxnuOTWxsrCIjI3XgwAGVlZXpmmuuUXx8vHbt2qWFCxdqypQpGjdunCQpMzNTqampmj17tlasWCGn06mSkhIVFhbKarUauXsAACAIGPrVc4vFcsbx5557TrfffrsaGhp022236YMPPlBra6tSUlKUm5urkpISn6mnzz77TAUFBdqyZYuio6OVn5+v5cuXKyyse12Or54DAND/dPfzO6ius2MUyg4AAP1Pv7zODgAAQKBRdgAAgKlRdgAAgKlRdgAAgKlRdgAAplFTU6NbbrlFNTU1RkdBEKHsAABMwe12a+XKlWpqatLKlSvldruNjoQgQdkBAJjCunXrdOTIEUnSkSNHVFZWZnAiBAvKDgCg3zt48KDKysp06tJxHo9HZWVlOnjwoMHJEAwoOwCAfs3j8Wj16tVnHefauaDsAAD6tfr6eu3YsUOdnZ0+452dndqxY4fq6+sNSoZgQdkBAPRrw4YN06RJkxQaGuozHhoaqksuuUTDhg0zKBmCBWUHANCvWSwWLViw4KzjZ7vpNAYOyg4AoN9LTk5WXl6et9hYLBbl5eXpvPPOMzgZggFlBwBgCrNmzVJ8fLwkKSEhQXl5eQYnQrCg7AAATMFms6moqEhJSUlauHChbDab0ZEQJMKMDgAAQKBcdtlluuyyy4yOgSDDkR0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqlB0AAGBqhpad0tJSTZo0STExMUpMTNQNN9ygffv2+azjdrtVWFio+Ph4DRo0SDNnzlRTU5PPOvX19crJyVFUVJQSExO1aNEidXR09OWuAACAIGVo2amurlZhYaG2b9+uyspKnThxQpmZmWptbfWus3DhQv35z3/Wyy+/rOrqajU2NurGG2/0Lu/s7FROTo7a29tVU1OjF154Qc8//7weeOABI3YJAAAEGYvH4/EYHeKUw4cPKzExUdXV1ZoyZYpaWlp07rnnqqysTDfddJMk6aOPPtKFF16o2tpaXXrppaqoqNC1116rxsZGJSUlSZKefvppLV68WIcPH1ZERMT3btflcik2NlYtLS2y2+29uo8AACAwuvv5HVTn7LS0tEiS4uLiJEk7d+7UiRMnNH36dO86Y8aM0bBhw1RbWytJqq2tVXp6urfoSFJWVpZcLpf27Nlzxu20tbXJ5XL5PAAAgDkFTdnp6urSvffeq8svv1xpaWmSJKfTqYiICA0ePNhn3aSkJDmdTu863y46p5afWnYmpaWlio2N9T5SUlICvDcAACBYBE3ZKSws1AcffKDy8vJe31ZxcbFaWlq8j4aGhl7fJgAAMEaY0QEkad68edqwYYO2bt2q5ORk77jD4VB7e7uOHj3qc3SnqalJDofDu84777zj836nvq11ap3/y2q1ymq1BngvAABAMDL0yI7H49G8efO0fv16vfnmmxo5cqTP8gkTJig8PFxVVVXesX379qm+vl4ZGRmSpIyMDO3evVvNzc3edSorK2W325Wamto3OwIAAIKWoUd2CgsLVVZWpldffVUxMTHec2xiY2MVGRmp2NhY3XnnnSoqKlJcXJzsdrvmz5+vjIwMXXrppZKkzMxMpaamavbs2VqxYoWcTqdKSkpUWFjI0RsAAGDsV88tFssZx5977jndfvvtkk5eVPC+++7TSy+9pLa2NmVlZenJJ5/0maL67LPPVFBQoC1btig6Olr5+flavny5wsK61+X46jkAAP1Pdz+/g+o6O0ah7AAA0P/0y+vsAAAABBplBwAAmBplBwAAmBplBwAAmBplBwAAmBplBwAAmBplBwAAmBplBwAAmBplBwAAmBplBwAAmBplBwAAmBplBwAAmBplBwAAmBplBwAAmBplBwBgGjU1NbrllltUU1NjdBQEEcoOAMAU3G63Vq5cqaamJq1cuVJut9voSAgSlB0AgCmsW7dOR44ckSQdOXJEZWVlBidCsKDsAAD6vYMHD6qsrEwej0eS5PF4VFZWpoMHDxqcDMGAsgMA6Nc8Ho9Wr1591vFTBQgDF2UHANCv1dfXa8eOHers7PQZ7+zs1I4dO1RfX29QMgQLyg4AoF8bNmyYJk2apNDQUJ/x0NBQXXLJJRo2bJhByRAsKDsAgH7NYrFowYIFZx23WCwGpEIwoewAAPq95ORk5eXleYuNxWJRXl6ezjvvPIOTIRhQdgAApjBr1izFx8dLkhISEpSXl2dwIgQLyg4AwBRsNpuKioqUlJSkhQsXymazGR0JQSLM6AAAAATKZZddpssuu8zoGAgyHNkBAACmRtkBAACmRtkBAACmRtkBAACmRtkBAACmRtkBAACmRtkBAACmRtkBAACmRtkBAACmxhWUJXk8HkmSy+UyOAkAAOiuU5/bpz7Hz4ayI+nYsWOSpJSUFIOTAACAf9SxY8cUGxt71uUWz/fVoQGgq6tLjY2NiomJkcViMToOepnL5VJKSooaGhpkt9uNjgMggPj7Hlg8Ho+OHTumoUOHKiTk7GfmcGRHUkhIiJKTk42OgT5mt9v5xxAwKf6+B47vOqJzCicoAwAAU6PsAAAAU6PsYMCxWq168MEHZbVajY4CIMD4+8aZcIIyAAAwNY7sAAAAU6PsAAAAU6PsAAAAU6PsAAAAU6PsYEBZs2aNRowYIZvNpsmTJ+udd94xOhKAANi6dauuu+46DR06VBaLRf/zP/9jdCQEEcoOBow//OEPKioq0oMPPqh3331XF110kbKystTc3Gx0NAA91Nraqosuukhr1qwxOgqCEF89x4AxefJkTZo0SU888YSkk/dES0lJ0fz583X//fcbnA5AoFgsFq1fv1433HCD0VEQJDiygwGhvb1dO3fu1PTp071jISEhmj59umpraw1MBgDobZQdDAh///vf1dnZqaSkJJ/xpKQkOZ1Og1IBAPoCZQcAAJgaZQcDQkJCgkJDQ9XU1OQz3tTUJIfDYVAqAEBfoOxgQIiIiNCECRNUVVXlHevq6lJVVZUyMjIMTAYA6G1hRgcA+kpRUZHy8/M1ceJEXXLJJVq1apVaW1t1xx13GB0NQA8dP35c+/fv9z7/5JNPVFdXp7i4OA0bNszAZAgGfPUcA8oTTzyh//zP/5TT6dT48eP1+OOPa/LkyUbHAtBDW7Zs0VVXXXXaeH5+vp5//vm+D4SgQtkBAACmxjk7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7AADA1Cg7ACCpvb3d6AgAegllB0BQ++Mf/6j09HRFRkYqPj5e06dPV2trqyTpd7/7ncaOHSur1aohQ4Zo3rx53tfV19drxowZGjRokOx2u3784x+rqanJu3zJkiUaP368fvvb32rkyJGy2WySpKNHj+pf/uVfdO6558put2vq1Kl6//33+3anAQQUZQdA0Dp06JBuvfVW/eQnP9GHH36oLVu26MYbb5TH49FTTz2lwsJC3X333dq9e7f+9Kc/afTo0ZKkrq4uzZgxQ1988YWqq6tVWVmpv/3tb7rlllt83n///v367//+b73yyiuqq6uTJN18881qbm5WRUWFdu7cqYsvvljTpk3TF1980de7DyBAuOs5gKD17rvvasKECfr00081fPhwn2XnnXee7rjjDj388MOnva6yslLZ2dn65JNPlJKSIknau3evxo4dq3feeUeTJk3SkiVLtGzZMn3++ec699xzJUnbtm1TTk6OmpubZbVave83evRo/du//ZvuvvvuXtxbAL0lzOgAAHA2F110kaZNm6b09HRlZWUpMzNTN910k06cOKHGxkZNmzbtjK/78MMPlZKS4i06kpSamqrBgwfrww8/1KRJkyRJw4cP9xYdSXr//fd1/PhxxcfH+7zf119/rQMHDvTCHgLoC5QdAEErNDRUlZWVqqmp0aZNm/TrX/9aP//5z1VVVRWQ94+OjvZ5fvz4cQ0ZMkRbtmw5bd3BgwcHZJsA+h5lB0BQs1gsuvzyy3X55ZfrgQce0PDhw1VZWakRI0aoqqpKV1111WmvufDCC9XQ0KCGhgafaayjR48qNTX1rNu6+OKL5XQ6FRYWphEjRvTWLgHoY5QdAEHr7bffVlVVlTIzM5WYmKi3335bhw8f1oUXXqglS5Zo7ty5SkxMVHZ2to4dO6a33npL8+fP1/Tp05Wenq5Zs2Zp1apV6ujo0E9/+lP98z//syZOnHjW7U2fPl0ZGRm64YYbtGLFCv3whz9UY2OjXnvtNeXm5n7nawEEL8oOgKBlt9u1detWrVq1Si6XS8OHD9ejjz6q7OxsSZLb7dZjjz2mn/3sZ0pISNBNN90k6eTRoFdffVXz58/XlClTFBISoh/96Ef69a9//Z3bs1gsev311/Xzn/9cd9xxhw4fPiyHw6EpU6YoKSmp1/cXQO/g21gAAMDUuM4OAAAwNcoOAAAwNcoOAAAwNcoOAAAwNcoOAAAwNcoOAAAwNcoOAAAwNcoOAAAwNcoOAAAwNcoOAAAwNcoOAAAwtf8PPHx+NBlWF8oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.boxplot(x='score', y='avg_chunk_len', data=pred_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7c23c763-f6cf-43b1-9659-8617ce1ddb8e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-04T04:19:00.595563Z",
     "iopub.status.busy": "2024-12-04T04:19:00.595422Z",
     "iopub.status.idle": "2024-12-04T04:19:00.662717Z",
     "shell.execute_reply": "2024-12-04T04:19:00.662260Z",
     "shell.execute_reply.started": "2024-12-04T04:19:00.595550Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='score', ylabel='max_chunk_len'>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAtDUlEQVR4nO3df1iVdZ7/8dcB5Bx/HUyUHybYrO5kJGia6dHRsUCIqDWzH5OmjNOP1UUzmTGHK8fMJnHpWtPKappp0tkk+7G5s2qk6KZNQWkYStq4q7WLhgdMk6MWIHC+f7Seb2fQosOB+/jx+biu+7o4n/vX+46LzsvP574/t83r9XoFAABgqDCrCwAAAGhPhB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKNFWF1AKGhublZVVZW6d+8um81mdTkAAKAVvF6vTp48qT59+igs7Pz9N4QdSVVVVUpISLC6DAAAEIBDhw6pb9++511P2JHUvXt3Sd/8x3I6nRZXAwAAWsPj8SghIcH3PX4+hB3JN3TldDoJOwAAXGC+7xYUblAGAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEazNOwsWrRINpvNbxk4cKBvfV1dnXJychQdHa1u3bpp0qRJqq6u9jtGZWWlsrKy1KVLF8XExGjevHlqbGzs6EsBAAAhyvLXRVx55ZXasmWL73NExP8vae7cudq4caNee+01RUVFadasWbrlllv03nvvSZKampqUlZWluLg4lZSU6MiRI5o2bZo6deqkJUuWdPi1AACA0GN52ImIiFBcXFyL9traWr3wwgsqLCzUddddJ0l68cUXdcUVV+j999/XyJEjtXnzZu3bt09btmxRbGyshgwZokcffVTz58/XokWLFBkZ2dGXAwAAQozlYee///u/1adPHzkcDrlcLuXn5ysxMVFlZWU6c+aM0tLSfNsOHDhQiYmJKi0t1ciRI1VaWqrk5GTFxsb6tsnIyNDMmTO1d+9eXXXVVec8Z319verr632fPR5P+12gIbxer+rq6qwuo828Xq/vd2+327/35XGhzuFwXPDXAADtzdKwM2LECK1atUqXX365jhw5okceeURjxozRxx9/LLfbrcjISPXo0cNvn9jYWLndbkmS2+32Czpn159ddz75+fl65JFHgnsxhqurq1NmZqbVZeBvFBUVqXPnzlaXAQAhzdKw8+0vz5SUFI0YMUL9+vXTq6++2q7/A8/Ly1Nubq7vs8fjUUJCQrudDwAAWMfyYaxv69Gjh3784x/rwIEDGj9+vBoaGnTixAm/3p3q6mrfPT5xcXHasWOH3zHOPq11rvuAzrLb7bLb7cG/AIM5HA4VFRVZXUab1dXVaeLEiZKkdevWyeFwWFxR21zo9QNARwipsHPq1CkdPHhQU6dO1bBhw9SpUydt3bpVkyZNkiTt379flZWVcrlckiSXy6XHHntMNTU1iomJkSQVFxfL6XQqKSnJsuswkc1mM264xOFwGHdNAICWLA07v/rVr3TTTTepX79+qqqq0sMPP6zw8HDdeeedioqK0t13363c3Fz17NlTTqdTs2fPlsvl0siRIyVJ6enpSkpK0tSpU1VQUCC3260FCxYoJyeHnhsAACDJ4rBz+PBh3XnnnTp27Jh69+6tn/zkJ3r//ffVu3dvSdITTzyhsLAwTZo0SfX19crIyNAzzzzj2z88PFwbNmzQzJkz5XK51LVrV2VnZ2vx4sVWXRIAAAgxNq/X67W6CKt5PB5FRUWptrZWTqfT6nLQjr7++mvfjfE8yQR8g6klQhNTS3y/1n5/h9Q9OwCAjsfUEqGJf5AFDy8CBQAARqNnBwAuckwtEZou9PpDCWEHAC5yTC0B0zGMBQAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjUkF25kpL9gzxbd/F/xeQgcvPATQngg77YwX7IWus9PKw3q88BBAe2IYCwAAGI2enQ50asid8obxn9xSXq/U3PjNz2EREkMnlrE1N6pb+ctWl9EmDFOHFoapQ1MoDFPzzduBvGERUngnq8uAIq0uAJK8VhcQBAxThy6GqUNHKAxTM4wFAACMRs8OAATB0z85Lnu4Cf1VFy6vV2po/ubnyDBGqa1U32TTrHd7Wl2GD2EHAILAHu6VPdzqKuCwugD8n9AK/gxjAQAAoxF2AACA0RjG6khNZ6yuAAgd/D0A6CCEnQ7Uffdaq0sAAOCiwzAWAAAwGj07Hejk4J8xqSBwVtMZejsBdAjCTkcK70TYAQCggzGMBQAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACj8W4sAAiC+iarKwBCR6j9PRB2ACAIZr0bbXUJAM6DYSwAAGA0enYAIAie/skx2cOtrgIIDfVNodXbSdgBgCCwh4uwA4QohrEAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjBYyYWfp0qWy2Wx64IEHfG3jxo2TzWbzW2bMmOG3X2VlpbKystSlSxfFxMRo3rx5amxs7ODqAQBAqIqwugBJ2rlzp373u98pJSWlxbp7771Xixcv9n3u0qWL7+empiZlZWUpLi5OJSUlOnLkiKZNm6ZOnTppyZIlHVI7AAAIbZb37Jw6dUpTpkzR73//e11yySUt1nfp0kVxcXG+xel0+tZt3rxZ+/bt00svvaQhQ4YoMzNTjz76qFauXKmGhoaOvAwAABCiLA87OTk5ysrKUlpa2jnXr1mzRr169dKgQYOUl5enr776yreutLRUycnJio2N9bVlZGTI4/Fo79695z1nfX29PB6P3wIAAMxk6TDW2rVrtWvXLu3cufOc6ydPnqx+/fqpT58+2rNnj+bPn6/9+/frjTfekCS53W6/oCPJ99ntdp/3vPn5+XrkkUeCdBUAACCUWRZ2Dh06pDlz5qi4uFgOh+Oc29x3332+n5OTkxUfH6/U1FQdPHhQ/fv3D/jceXl5ys3N9X32eDxKSEgI+HgAACB0WTaMVVZWppqaGg0dOlQRERGKiIjQ9u3b9eSTTyoiIkJNTU0t9hkxYoQk6cCBA5KkuLg4VVdX+21z9nNcXNx5z2232+V0Ov0WAABgJsvCTmpqqioqKlReXu5brr76ak2ZMkXl5eUKDw9vsU95ebkkKT4+XpLkcrlUUVGhmpoa3zbFxcVyOp1KSkrqkOsAAAChzbJhrO7du2vQoEF+bV27dlV0dLQGDRqkgwcPqrCwUDfccIOio6O1Z88ezZ07V2PHjvU9op6enq6kpCRNnTpVBQUFcrvdWrBggXJycmS32624LAAAEGJCYp6dc4mMjNSWLVu0fPlynT59WgkJCZo0aZIWLFjg2yY8PFwbNmzQzJkz5XK51LVrV2VnZ/vNyxNKbM2N8lpdxMXO65Wa/2/SybAIyWaztp6LmK2ZyT8BdIyQCjvbtm3z/ZyQkKDt27d/7z79+vXTm2++2Y5VBU+38petLgEAgIuO5fPsAAAAtKeQ6tkxkcPhUFFRkdVl4P/U1dVp4sSJkqR169add9oDdCx+DwDaE2GnndlsNnXu3NnqMnAODoeD3w0AXAQYxgIAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBoEVYXAAAmqG+ySfJaXcZFzeuVGpq/+TkyTLLZrK3nYvbN30PoIOwAQBDMeren1SUAOA+GsQAAgNHo2QGAADkcDhUVFVldBv5PXV2dJk6cKElat26dHA6HxRVBUkj8Hgg7ABAgm82mzp07W10GzsHhcPC7gQ/DWAAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjMbTWGgVr9eruro6q8tos29fgwnX43A4ZGOaWAD4ToQdtEpdXZ0yMzOtLiOozs7HcSErKiri8VoA+B4MYwEAAKPRs4NWMWWmWK/Xq/r6ekmS3W6/4IeAQmFmUgAIdYQdtIpJM8V26dLF6hIAAB2IYSwAAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNECnmfnxIkT2rFjh2pqatTc3Oy3btq0aW0uDAAAIBgCCjvr16/XlClTdOrUKTmdTr9ZaG02G2EHAACEjICGsX75y1/qF7/4hU6dOqUTJ07oyy+/9C3Hjx8Pdo0AAAABCyjsfP7557r//vuZdh8AAIS8gMJORkaGPvzww2DXAgAAEHQB3bOTlZWlefPmad++fUpOTlanTp381v/DP/xDUIoDAABoq4DCzr333itJWrx4cYt1NptNTU1NbasKAAAgSAIKO3/7qDkAAECoavOkgnV1dcGoAwAAoF0EFHaampr06KOP6tJLL1W3bt306aefSpJ+85vf6IUXXghqgQAAAG0RUNh57LHHtGrVKhUUFCgyMtLXPmjQIP3hD38IWnEAAABtFVDY+dOf/qTnn39eU6ZMUXh4uK998ODB+utf/xq04gAAANoq4EkFBwwY0KK9ublZZ86caXNRAAAAwRJQ2ElKStJf/vKXFu2vv/66rrrqqoAKWbp0qWw2mx544AFfW11dnXJychQdHa1u3bpp0qRJqq6u9tuvsrJSWVlZ6tKli2JiYjRv3jw1NjYGVAMAADBPQI+eL1y4UNnZ2fr888/V3NysN954Q/v379ef/vQnbdiw4Qcfb+fOnfrd736nlJQUv/a5c+dq48aNeu211xQVFaVZs2bplltu0XvvvSfpmxuls7KyFBcXp5KSEh05ckTTpk1Tp06dtGTJkkAuDQAAGCagnp0JEyZo/fr12rJli7p27aqFCxfqk08+0fr16zV+/PgfdKxTp05pypQp+v3vf69LLrnE115bW6sXXnhBy5Yt03XXXadhw4bpxRdfVElJid5//31J0ubNm7Vv3z699NJLGjJkiDIzM/Xoo49q5cqVamhoOO856+vr5fF4/BYAAGCmgOfZGTNmjIqLi1VTU6OvvvpK7777rtLT03/wcXJycpSVlaW0tDS/9rKyMp05c8avfeDAgUpMTFRpaakkqbS0VMnJyYqNjfVtk5GRIY/Ho7179573nPn5+YqKivItCQkJP7huAABwYWjzpIJtsXbtWu3atUv5+fkt1rndbkVGRqpHjx5+7bGxsXK73b5tvh10zq4/u+588vLyVFtb61sOHTrUxisBAAChqtX37FxyySWy2Wyt2vb48ePfu82hQ4c0Z84cFRcXy+FwtLaMoLDb7bLb7R16TgAAYI1Wh53ly5cH9cRlZWWqqanR0KFDfW1NTU1655139PTTT2vTpk1qaGjQiRMn/Hp3qqurFRcXJ0mKi4vTjh07/I579mmts9sAAICLW6vDTnZ29g8++NKlSzVjxowWQ1GSlJqaqoqKCr+26dOna+DAgZo/f74SEhLUqVMnbd26VZMmTZIk7d+/X5WVlXK5XJIkl8ulxx57TDU1NYqJiZEkFRcXy+l0Kikp6QfXCwAAzBPQo+ettWTJEt1+++3nDDvdu3fXoEGD/Nq6du2q6OhoX/vdd9+t3Nxc9ezZU06nU7Nnz5bL5dLIkSMlSenp6UpKStLUqVNVUFAgt9utBQsWKCcnh2EqAAAgqZ3DjtfrbdP+TzzxhMLCwjRp0iTV19crIyNDzzzzjG99eHi4NmzYoJkzZ8rlcqlr167Kzs7W4sWL21o6AAAwhM3b1kTyHbp3767du3fr7/7u79rrFEHh8XgUFRWl2tpaOZ1Oq8sBAATg66+/VmZmpiSpqKhInTt3trgitLfWfn9b+ug5AABAeyPsAAAAoxF2AACA0YIedr7++mvfz2PGjGHMFAAAWCqgsHP//fefs/306dO64YYbfJ/ffPNNxcfHB1YZAABAEAQUdjZu3KiHH37Yr+306dO6/vrr1djYGJTCAAAAgiGgeXY2b96sMWPG6JJLLtEDDzygkydPKiMjQxERESoqKgp2jQAAAAELKOz0799fb731lq699lqFhYXp5Zdflt1u18aNG9W1a9dg1wgAABCwgGdQTklJ0YYNGzR+/HiNGDFCGzZs4GZkAAAQcloddq666irZbLYW7Xa7XVVVVRo9erSvbdeuXcGpDgAAoI1aHXZuvvnmdiwDAACgfbQ67Pzt01cAAAAXgja99byhoUE1NTVqbm72a09MTGxTUQAAAMESUNj5r//6L919990qKSnxa/d6vbLZbGpqagpKcQAAAG0VUNiZPn26IiIitGHDBsXHx5/zxmUgVJWUlGjFihWaM2eORo0aZXU5AIB2FlDYKS8vV1lZmQYOHBjseoB2VVdXp2XLlumLL77QsmXLNHToUDkcDqvLAgC0o4BeF5GUlKQvvvgi2LUA7W7NmjU6duyYJOnYsWMqLCy0uCIAQHsLKOz88z//sx588EFt27ZNx44dk8fj8VuAUHT48GEVFhbK6/VK+uYes8LCQh0+fNjiygAA7SmgYay0tDRJUmpqql87NygjVHm9Xq1YseK87QUFBdx7BgCGCijsvP3228GuA2hXlZWV2rlzZ4v2pqYm7dy5U5WVlerXr58FlQEA2ltAYeenP/1psOsA2lViYqKGDx+uXbt2+fU8hoeHa9iwYcwNBQAGCyjsvPPOO9+5fuzYsQEVA7QXm82mOXPmaNq0aS3WzZkzhyEsADBYQGFn3LhxLdq+/WXBPTsIRX379tWVV16piooKX9uVV16pSy+91MKqAADtLaCnsb788ku/paamRm+99ZaGDx+uzZs3B7tGICgOHz6sffv2+bXt27ePp7EAwHAB9exERUW1aBs/frwiIyOVm5ursrKyNhcGBBNPYwHAxSugnp3ziY2N1f79+4N5SCAozj6N9bdDrN9+GgsAYKaAenb27Nnj99nr9erIkSNaunSphgwZEoy6gKDiaSwAuHgFFHaGDBkim83mm4n2rJEjR+qPf/xjUAoDguns01jZ2dnnbGcICwDMFVDY+eyzz/w+h4WFqXfv3rxQESGtb9++mjx5sl566SXfbN+TJ0/maSwAMFxAYYeZZnGhmjJlioqKivTFF1+oV69emjx5stUlAQDaWUBhR5K2bt2qrVu3qqamRs3NzX7rGMpCqHI4HMrNzdWKFSs0Z84ceiMB4CIQUNh55JFHtHjxYl199dWKj4/nfgdcUEaNGqVRo0ZZXQYAoIMEFHaee+45rVq1SlOnTg12PQAAAEEV0Dw7DQ0N/MsYAABcEAIKO/fcc48KCwuDXQsAAEDQtXoYKzc31/dzc3Oznn/+eW3ZskUpKSnq1KmT37bLli0LXoUAAABt0Oqw89FHH/l9PjtT8scff+zXzs3KAAAglLQ67Lz99tvtWQcAAEC7COiendraWh0/frxF+/Hjx+XxeNpcFAAAQLAEFHZ+9rOfae3atS3aX331Vf3sZz9rc1EAAADBElDY+eCDD3Tttde2aB83bpw++OCDNhcFAAAQLAGFnfr6ejU2NrZoP3PmjL7++us2FwUAABAsAYWda665Rs8//3yL9ueee07Dhg1rc1EAAADBEtDrIn77298qLS1Nu3fvVmpqqqRvXgy6c+dObd68OagFAgAAtEVAPTujR49WaWmpEhIS9Oqrr2r9+vUaMGCA9uzZozFjxgS7RgAAgIAF1LMjfTOp4Jo1a75zm6VLl2rGjBnq0aNHoKcBAABok4B6dlpryZIl55yPBwAAoKO0a9jxer3teXgAAIDv1a5hBwAAwGoB37MDADCD1+tVXV2d1WW02bevwYTrcTgcvFw7SAg7AHCRq6urU2ZmptVlBNXEiROtLqHNioqK1LlzZ6vLMALDWAAAwGjt2rMzZswYUikAhDiHw6GioiKry2gzr9er+vp6SZLdbr/gh4AcDofVJRgjoLCzatUq/fznP2/R3tjYqN/85jfKz8+XJL355pttKg4A0P5sNpsx/zDt0qWL1SUgBAU0jHX//ffrtttu05dffulr279/v0aMGKGXX345aMUBAAC0VUBh56OPPtLhw4eVnJys4uJirVy5UkOHDtXAgQO1e/fuYNcIAAAQsICGsfr376/33ntPDzzwgK6//nqFh4dr9erVuvPOO4NdHwAAQJsE/DTWxo0btXbtWrlcLvXo0UMvvPCCqqqqftAxnn32WaWkpMjpdMrpdMrlcvndJDdu3DjZbDa/ZcaMGX7HqKysVFZWlrp06aKYmBjNmzdPjY2NgV4WAAAwTEBh5x//8R912223af78+frLX/6iPXv2KDIyUsnJyXr11VdbfZy+fftq6dKlKisr04cffqjrrrtOEyZM0N69e33b3HvvvTpy5IhvKSgo8K1rampSVlaWGhoaVFJSotWrV2vVqlVauHBhIJcFAAAMZPMG8AKrQYMGac2aNRo8eLBf+8qVKzV//nydOnUq4IJ69uypxx9/XHfffbfGjRunIUOGaPny5efctqioSDfeeKOqqqoUGxsrSXruuec0f/58HT16VJGRkefcr76+3vd4oiR5PB4lJCSotrZWTqcz4NoBAEDH8Xg8ioqK+t7v74B6dsrKyloEHUnKyclRWVlZIIdUU1OT1q5dq9OnT8vlcvna16xZo169emnQoEHKy8vTV1995VtXWlqq5ORkX9CRpIyMDHk8Hr/eob+Vn5+vqKgo35KQkBBQzQAAIPQFdIOy3W4/77rLL7/8Bx2roqJCLpdLdXV16tatm9atW6ekpCRJ0uTJk9WvXz/16dNHe/bs0fz587V//3698cYbkiS32+0XdCT5Prvd7vOeMy8vT7m5ub7PZ3t2AACAeQKeQfn111/Xq6++qsrKSjU0NPit27VrV6uPc/nll6u8vFy1tbV6/fXXlZ2dre3btyspKUn33Xefb7vk5GTFx8crNTVVBw8eVP/+/QMtXXa7/TsDGwAAMEdAw1hPPvmkpk+frtjYWH300Ue65pprFB0drU8//fQHv0wuMjJSAwYM0LBhw5Sfn6/BgwdrxYoV59x2xIgRkqQDBw5IkuLi4lRdXe23zdnPcXFxP/SyAACAgQIKO88884yef/55PfXUU4qMjNSDDz6o4uJi3X///aqtrW1TQc3NzX43D39beXm5JCk+Pl6S5HK5VFFRoZqaGt82xcXFcjqdvqEwAABwcQtoGKuyslKjRo2SJHXu3FknT56UJE2dOlUjR47U008/3arj5OXlKTMzU4mJiTp58qQKCwu1bds2bdq0SQcPHlRhYaFuuOEGRUdHa8+ePZo7d67Gjh2rlJQUSVJ6erqSkpI0depUFRQUyO12a8GCBcrJyWGYCgAASAqwZycuLk7Hjx+XJCUmJur999+XJH322Wf6IU+y19TUaNq0abr88suVmpqqnTt3atOmTRo/frwiIyO1ZcsWpaena+DAgfrlL3+pSZMmaf369b79w8PDtWHDBoWHh8vlcumuu+7StGnTtHjx4kAuCwAAGCigeXbuueceJSQk6OGHH9bKlSs1b948jR49Wh9++KFuueUWvfDCC+1Ra7tp7XP6AAAgdLT2+zugsNPc3Kzm5mZFRHwzCvbKK6/ovffe09///d9rxowZ6tSpU+CVW4CwAwDAhaddw44k1dXVac+ePaqpqVFzc/P/P6DNpptuuimQQ1qGsAMAwIWntd/fAd2g/NZbb2nq1Kk6duxYi3U2m01NTU2BHBYAACDoArpBefbs2br99tt15MgR35DW2YWgAwAAQklAYae6ulq5ubktXtUAAAAQagIKO7feequ2bdsW5FIAAACCL6AblL/66ivddttt6t27t5KTk1s8fXX//fcHrcCOwA3KAABceNr1BuWXX35ZmzdvlsPh0LZt22Sz2XzrbDbbBRd2AACAuQIKOw899JAeeeQR/frXv1ZYWEAjYQAAAB0ioKTS0NCgO+64g6ADAABCXkBpJTs7W6+88kqwawEAAAi6gIaxmpqaVFBQoE2bNiklJaXFDcrLli0LSnEAAABtFVDYqaio0FVXXSVJ+vjjj/3WfftmZQAAAKsFFHbefvvtYNcBAADQLrjDGAAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEazNOw8++yzSklJkdPplNPplMvlUlFRkW99XV2dcnJyFB0drW7dumnSpEmqrq72O0ZlZaWysrLUpUsXxcTEaN68eWpsbOzoSwEAACHK0rDTt29fLV26VGVlZfrwww913XXXacKECdq7d68kae7cuVq/fr1ee+01bd++XVVVVbrlllt8+zc1NSkrK0sNDQ0qKSnR6tWrtWrVKi1cuNCqSwIAACHG5vV6vVYX8W09e/bU448/rltvvVW9e/dWYWGhbr31VknSX//6V11xxRUqLS3VyJEjVVRUpBtvvFFVVVWKjY2VJD333HOaP3++jh49qsjIyFad0+PxKCoqSrW1tXI6ne12bQAAIHha+/0dMvfsNDU1ae3atTp9+rRcLpfKysp05swZpaWl+bYZOHCgEhMTVVpaKkkqLS1VcnKyL+hIUkZGhjwej6936Fzq6+vl8Xj8FgAAYCbLw05FRYW6desmu92uGTNmaN26dUpKSpLb7VZkZKR69Ojht31sbKzcbrckye12+wWds+vPrjuf/Px8RUVF+ZaEhITgXhQAAAgZloedyy+/XOXl5frggw80c+ZMZWdna9++fe16zry8PNXW1vqWQ4cOtev5AACAdSKsLiAyMlIDBgyQJA0bNkw7d+7UihUrdMcdd6ihoUEnTpzw692prq5WXFycJCkuLk47duzwO97Zp7XObnMudrtddrs9yFcCAABCkeU9O3+rublZ9fX1GjZsmDp16qStW7f61u3fv1+VlZVyuVySJJfLpYqKCtXU1Pi2KS4ultPpVFJSUofXDgAAQo+lPTt5eXnKzMxUYmKiTp48qcLCQm3btk2bNm1SVFSU7r77buXm5qpnz55yOp2aPXu2XC6XRo4cKUlKT09XUlKSpk6dqoKCArndbi1YsEA5OTn03AAAAEkWh52amhpNmzZNR44cUVRUlFJSUrRp0yaNHz9ekvTEE08oLCxMkyZNUn19vTIyMvTMM8/49g8PD9eGDRs0c+ZMuVwude3aVdnZ2Vq8eLFVlwQAAEJMyM2zYwXm2QEA4MJzwc2zAwAA0B4IOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwBgjJKSEt1xxx0qKSmxuhSEEMIOAMAIdXV1WrZsmaqrq7Vs2TLV1dVZXRJCBGEHAGCENWvW6NixY5KkY8eOqbCw0OKKECoIOwCAC97hw4dVWFgor9crSfJ6vSosLNThw4ctrgyhgLADALigeb1erVix4rztZwMQLl6Whp38/HwNHz5c3bt3V0xMjG6++Wbt37/fb5tx48bJZrP5LTNmzPDbprKyUllZWerSpYtiYmI0b948NTY2duSlAAAsUllZqZ07d6qpqcmvvampSTt37lRlZaVFlSFUWBp2tm/frpycHL3//vsqLi7WmTNnlJ6ertOnT/ttd++99+rIkSO+paCgwLeuqalJWVlZamhoUElJiVavXq1Vq1Zp4cKFHX05AAALJCYmavjw4QoPD/drDw8P1zXXXKPExESLKkOosHlDqH/v6NGjiomJ0fbt2zV27FhJ3/TsDBkyRMuXLz/nPkVFRbrxxhtVVVWl2NhYSdJzzz2n+fPn6+jRo4qMjPze83o8HkVFRam2tlZOpzNo1wMA6BiHDx9Wdna2X+9ORESEVq9erUsvvdTCytCeWvv9HVL37NTW1kqSevbs6de+Zs0a9erVS4MGDVJeXp6++uor37rS0lIlJyf7go4kZWRkyOPxaO/evec8T319vTwej98CALhw9e3bV5MnT5bNZpMk2Ww2TZ48maADSSEUdpqbm/XAAw9o9OjRGjRokK998uTJeumll/T2228rLy9P//qv/6q77rrLt97tdvsFHUm+z263+5znys/PV1RUlG9JSEhohysCAHSkKVOmKDo6WpLUq1cvTZ482eKKECoirC7grJycHH388cd69913/drvu+8+38/JycmKj49XamqqDh48qP79+wd0rry8POXm5vo+ezweAg8AXOAcDodyc3O1YsUKzZkzRw6Hw+qSECJCIuzMmjVLGzZs0DvvvKO+fft+57YjRoyQJB04cED9+/dXXFycduzY4bdNdXW1JCkuLu6cx7Db7bLb7UGoHAAQSkaNGqVRo0ZZXQZCjKXDWF6vV7NmzdK6dev0n//5n/rRj370vfuUl5dLkuLj4yVJLpdLFRUVqqmp8W1TXFwsp9OppKSkdqkbAABcOCzt2cnJyVFhYaH+/Oc/q3v37r57bKKiotS5c2cdPHhQhYWFuuGGGxQdHa09e/Zo7ty5Gjt2rFJSUiRJ6enpSkpK0tSpU1VQUCC3260FCxYoJyeH3hsAAGDto+dn75r/Wy+++KJ+/vOf69ChQ7rrrrv08ccf6/Tp00pISNDEiRO1YMECv0fM/vd//1czZ87Utm3b1LVrV2VnZ2vp0qWKiGhdluPRcwAALjyt/f4OqXl2rELYAQDgwnNBzrMDAAAQbIQdAABgNMIOAAAwGmEHAAAYjbADAACMFhIzKFvt7ANpvBAUAIALx9nv7e97sJywI+nkyZOSxPuxAAC4AJ08eVJRUVHnXc88O/rmjetVVVXq3r37eSc6hDnOvvj10KFDzKsEGIa/74uL1+vVyZMn1adPH4WFnf/OHHp2JIWFhX3vC0hhHqfTyf8MAUPx933x+K4enbO4QRkAABiNsAMAAIxG2MFFx2636+GHH5bdbre6FABBxt83zoUblAEAgNHo2QEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHVxUVq5cqcsuu0wOh0MjRozQjh07rC4JQBC88847uummm9SnTx/ZbDb9+7//u9UlIYQQdnDReOWVV5Sbm6uHH35Yu3bt0uDBg5WRkaGamhqrSwPQRqdPn9bgwYO1cuVKq0tBCOLRc1w0RowYoeHDh+vpp5+W9M070RISEjR79mz9+te/trg6AMFis9m0bt063XzzzVaXghBBzw4uCg0NDSorK1NaWpqvLSwsTGlpaSotLbWwMgBAeyPs4KLwxRdfqKmpSbGxsX7tsbGxcrvdFlUFAOgIhB0AAGA0wg4uCr169VJ4eLiqq6v92qurqxUXF2dRVQCAjkDYwUUhMjJSw4YN09atW31tzc3N2rp1q1wul4WVAQDaW4TVBQAdJTc3V9nZ2br66qt1zTXXaPny5Tp9+rSmT59udWkA2ujUqVM6cOCA7/Nnn32m8vJy9ezZU4mJiRZWhlDAo+e4qDz99NN6/PHH5Xa7NWTIED355JMaMWKE1WUBaKNt27bp2muvbdGenZ2tVatWdXxBCCmEHQAAYDTu2QEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AEBSQ0OD1SUAaCeEHQAh7fXXX1dycrI6d+6s6OhopaWl6fTp05KkP/7xj7ryyitlt9sVHx+vWbNm+farrKzUhAkT1K1bNzmdTt1+++2qrq72rV+0aJGGDBmiP/zhD/rRj34kh8MhSTpx4oTuuece9e7dW06nU9ddd512797dsRcNIKgIOwBC1pEjR3TnnXfqF7/4hT755BNt27ZNt9xyi7xer5599lnl5OTovvvuU0VFhf7jP/5DAwYMkCQ1NzdrwoQJOn78uLZv367i4mJ9+umnuuOOO/yOf+DAAf3bv/2b3njjDZWXl0uSbrvtNtXU1KioqEhlZWUaOnSoUlNTdfz48Y6+fABBwlvPAYSsXbt2adiwYfqf//kf9evXz2/dpZdequnTp+u3v/1ti/2Ki4uVmZmpzz77TAkJCZKkffv26corr9SOHTs0fPhwLVq0SEuWLNHnn3+u3r17S5LeffddZWVlqaamRna73Xe8AQMG6MEHH9R9993XjlcLoL1EWF0AAJzP4MGDlZqaquTkZGVkZCg9PV233nqrzpw5o6qqKqWmpp5zv08++UQJCQm+oCNJSUlJ6tGjhz755BMNHz5cktSvXz9f0JGk3bt369SpU4qOjvY73tdff62DBw+2wxUC6AiEHQAhKzw8XMXFxSopKdHmzZv11FNP6aGHHtLWrVuDcvyuXbv6fT516pTi4+O1bdu2Ftv26NEjKOcE0PEIOwBCms1m0+jRozV69GgtXLhQ/fr1U3FxsS677DJt3bpV1157bYt9rrjiCh06dEiHDh3yG8Y6ceKEkpKSznuuoUOHyu12KyIiQpdddll7XRKADkbYARCyPvjgA23dulXp6emKiYnRBx98oKNHj+qKK67QokWLNGPGDMXExCgzM1MnT57Ue++9p9mzZystLU3JycmaMmWKli9frsbGRv3TP/2TfvrTn+rqq68+7/nS0tLkcrl08803q6CgQD/+8Y9VVVWljRs3auLEid+5L4DQRdgBELKcTqfeeecdLV++XB6PR/369dO//Mu/KDMzU5JUV1enJ554Qr/61a/Uq1cv3XrrrZK+6Q3685//rNmzZ2vs2LEKCwvT9ddfr6eeeuo7z2ez2fTmm2/qoYce0vTp03X06FHFxcVp7Nixio2NbffrBdA+eBoLAAAYjXl2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGC0/wf51+w7rKvIJwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.boxplot(x='score', y='max_chunk_len', data=pred_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "171db568-b905-40da-8f15-50f0f07270f7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1058.563616,
   "end_time": "2024-11-23T14:46:37.625874",
   "environment_variables": {},
   "exception": null,
   "input_path": "13_contextual_embeddings.ipynb",
   "output_path": "run_13_contextual_embeddings.ipynb",
   "parameters": {},
   "start_time": "2024-11-23T14:28:59.062258",
   "version": "2.6.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0cd8c168767249f2a5fa412173f6e751": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_5ce1d1d9d86c40d9839877ff95734491",
       "max": 100,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_231702cf4d79477f9d5548665a1b18fe",
       "tabbable": null,
       "tooltip": null,
       "value": 100
      }
     },
     "2133bb8d85d34b8db112b4408ad60320": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "231702cf4d79477f9d5548665a1b18fe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "23b1ad9c0f9c46c888da66e85c90eb84": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "24e6eadc3dc940ecabf30dd1a3c6d1f3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_fa4bddf2c33241b5bf918054518f128f",
       "max": 52,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_edc33e82be8f41eba6a18a0ef074ab7a",
       "tabbable": null,
       "tooltip": null,
       "value": 52
      }
     },
     "2f60367b1c8941e2bf71661c33969ae8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3865f25c78aa46f29a25d807205281c3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3d0b06deaa654b989eece8cde06fa0f8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "3f8ceda83287475b97608e42f5f6782f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4881e496f1c84fe29ce9ebebaddfb3c2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_bd096d5d219a467786a85cfe1613fedd",
        "IPY_MODEL_24e6eadc3dc940ecabf30dd1a3c6d1f3",
        "IPY_MODEL_bc2b8104b4244d8cacedeb95e800d91c"
       ],
       "layout": "IPY_MODEL_6b9a8e43c1c342dba500a14e7149b600",
       "tabbable": null,
       "tooltip": null
      }
     },
     "5ce1d1d9d86c40d9839877ff95734491": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5ddb08be5cc64c9ab40a1d62a21763a5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_86283159049d48b1adcfb2de2d404d4d",
       "placeholder": "​",
       "style": "IPY_MODEL_2133bb8d85d34b8db112b4408ad60320",
       "tabbable": null,
       "tooltip": null,
       "value": " 100/100 [08:34&lt;00:00, 10.01s/it]"
      }
     },
     "5ef9d83ccad1471f85335900a24a8553": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6b9a8e43c1c342dba500a14e7149b600": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "816a079a8c804fbfa9b9a74f941abea8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_bcc69ec5db1b4aab977807284c9290e7",
        "IPY_MODEL_0cd8c168767249f2a5fa412173f6e751",
        "IPY_MODEL_5ddb08be5cc64c9ab40a1d62a21763a5"
       ],
       "layout": "IPY_MODEL_d1178c6858284f788a80b5f2a14fd0b7",
       "tabbable": null,
       "tooltip": null
      }
     },
     "86283159049d48b1adcfb2de2d404d4d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8ff8262c56604119883f4a5f13bb74ab": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_5ef9d83ccad1471f85335900a24a8553",
       "placeholder": "​",
       "style": "IPY_MODEL_e89e77133c344fc48c1d62f5a607ec93",
       "tabbable": null,
       "tooltip": null,
       "value": " 8/8 [00:18&lt;00:00,  2.27s/it]"
      }
     },
     "9189a076554543aaa6f5ee04e40dbe1b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "988e6697a2af486fadeaf0b84347b565": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_e1aae4c55cb64f379e74f15357275628",
        "IPY_MODEL_fd9e23198ca1489a9773fda3510bf857",
        "IPY_MODEL_8ff8262c56604119883f4a5f13bb74ab"
       ],
       "layout": "IPY_MODEL_d2ee15001d2244529f7e47d3333c0f8e",
       "tabbable": null,
       "tooltip": null
      }
     },
     "9fc7d91f94a94933bde5ba80e64587de": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "a7d240a289084bdfba4724c0efd5ab07": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "bc2b8104b4244d8cacedeb95e800d91c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_2f60367b1c8941e2bf71661c33969ae8",
       "placeholder": "​",
       "style": "IPY_MODEL_9fc7d91f94a94933bde5ba80e64587de",
       "tabbable": null,
       "tooltip": null,
       "value": " 52/52 [04:26&lt;00:00,  4.22s/it]"
      }
     },
     "bcc69ec5db1b4aab977807284c9290e7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_3f8ceda83287475b97608e42f5f6782f",
       "placeholder": "​",
       "style": "IPY_MODEL_3d0b06deaa654b989eece8cde06fa0f8",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "bd096d5d219a467786a85cfe1613fedd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_3865f25c78aa46f29a25d807205281c3",
       "placeholder": "​",
       "style": "IPY_MODEL_9189a076554543aaa6f5ee04e40dbe1b",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "cc3ed8dc4a5c43aca7b62d904865b2fa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cf68b6fe24964ce792aa63827489cb97": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "d1178c6858284f788a80b5f2a14fd0b7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d2ee15001d2244529f7e47d3333c0f8e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e1aae4c55cb64f379e74f15357275628": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_23b1ad9c0f9c46c888da66e85c90eb84",
       "placeholder": "​",
       "style": "IPY_MODEL_cf68b6fe24964ce792aa63827489cb97",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "e89e77133c344fc48c1d62f5a607ec93": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "edc33e82be8f41eba6a18a0ef074ab7a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "fa4bddf2c33241b5bf918054518f128f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fd9e23198ca1489a9773fda3510bf857": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_cc3ed8dc4a5c43aca7b62d904865b2fa",
       "max": 8,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_a7d240a289084bdfba4724c0efd5ab07",
       "tabbable": null,
       "tooltip": null,
       "value": 8
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
